{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('id_gsd-ud-train.conllu.txt') as fopen:\n",
    "    corpus = fopen.read().split('\\n')\n",
    "    \n",
    "with open('id_gsd-ud-test.conllu.txt') as fopen:\n",
    "    corpus.extend(fopen.read().split('\\n'))\n",
    "    \n",
    "with open('id_gsd-ud-dev.conllu.txt') as fopen:\n",
    "    corpus.extend(fopen.read().split('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {'PAD': 0,'NUM':1,'UNK':2}\n",
    "tag2idx = {'PAD': 0}\n",
    "char2idx = {'PAD': 0,'NUM':1,'UNK':2}\n",
    "word_idx = 3\n",
    "tag_idx = 1\n",
    "char_idx = 3\n",
    "\n",
    "def process_string(string):\n",
    "    string = re.sub('[^A-Za-z0-9\\-\\/ ]+', ' ', string).split()\n",
    "    return [to_title(y.strip()) for y in string]\n",
    "\n",
    "def to_title(string):\n",
    "    if string.isupper():\n",
    "        string = string.title()\n",
    "    return string\n",
    "\n",
    "def process_corpus(corpus, until = None):\n",
    "    global word2idx, tag2idx, char2idx, word_idx, tag_idx, char_idx\n",
    "    sentences, words, depends, labels, pos = [], [], [], [], []\n",
    "    temp_sentence, temp_word, temp_depend, temp_label, temp_pos = [], [], [], [], []\n",
    "    for sentence in corpus:\n",
    "        if len(sentence):\n",
    "            if sentence[0] == '#':\n",
    "                continue\n",
    "            sentence = sentence.split('\\t')\n",
    "            temp = process_string(sentence[1])\n",
    "            if not len(temp):\n",
    "                sentence[1] = 'EMPTY'\n",
    "            sentence[1] = process_string(sentence[1])[0]\n",
    "            for c in sentence[1]:\n",
    "                if c not in char2idx:\n",
    "                    char2idx[c] = char_idx\n",
    "                    char_idx += 1\n",
    "            if sentence[7] not in tag2idx:\n",
    "                tag2idx[sentence[7]] = tag_idx\n",
    "                tag_idx += 1\n",
    "            if sentence[1] not in word2idx:\n",
    "                word2idx[sentence[1]] = word_idx\n",
    "                word_idx += 1\n",
    "            temp_word.append(word2idx[sentence[1]])\n",
    "            temp_depend.append(int(sentence[6]) + 1)\n",
    "            temp_label.append(tag2idx[sentence[7]])\n",
    "            temp_sentence.append(sentence[1])\n",
    "            temp_pos.append(sentence[3])\n",
    "        else:\n",
    "            words.append(temp_word)\n",
    "            depends.append(temp_depend)\n",
    "            labels.append(temp_label)\n",
    "            sentences.append(temp_sentence)\n",
    "            pos.append(temp_pos)\n",
    "            temp_word = []\n",
    "            temp_depend = []\n",
    "            temp_label = []\n",
    "            temp_sentence = []\n",
    "            temp_pos = []\n",
    "    return sentences[:-1], words[:-1], depends[:-1], labels[:-1], pos[:-1]\n",
    "        \n",
    "sentences, words, depends, labels, pos = process_corpus(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('augmented.json') as fopen:\n",
    "    augmented = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_XY(texts):\n",
    "    global word2idx, tag2idx, char2idx, word_idx, tag_idx, char_idx\n",
    "    outside, sentences = [], []\n",
    "    for no, text in enumerate(texts):\n",
    "        s = process_string(text)\n",
    "        sentences.append(s)\n",
    "        inside = []\n",
    "        for w in s:\n",
    "            for c in w:\n",
    "                if c not in char2idx:\n",
    "                    char2idx[c] = char_idx\n",
    "                    char_idx += 1\n",
    "            \n",
    "            if w not in word2idx:\n",
    "                word2idx[w] = word_idx\n",
    "                word_idx += 1\n",
    "                \n",
    "            inside.append(word2idx[w])\n",
    "        outside.append(inside)\n",
    "    return outside, sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_augmented = []\n",
    "for a in augmented:\n",
    "    text_augmented.extend(a[0])\n",
    "    depends.extend(a[1])\n",
    "    labels.extend(a[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "outside, new_sentences = parse_XY(text_augmented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "words.extend(outside)\n",
    "sentences.extend(new_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50365, 50365, 50365, 50365)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words), len(depends), len(labels), len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_char_seq(batch, UNK = 2):\n",
    "    maxlen_c = max([len(k) for k in batch])\n",
    "    x = [[len(i) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((len(batch),maxlen_c,maxlen),dtype=np.int32)\n",
    "    for i in range(len(batch)):\n",
    "        for k in range(len(batch[i])):\n",
    "            for no, c in enumerate(batch[i][k][:maxlen][::-1]):\n",
    "                temp[i,k,-1-no] = char2idx.get(c, UNK)\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx2word = {idx: tag for tag, idx in word2idx.items()}\n",
    "idx2tag = {i: w for w, i in tag2idx.items()}\n",
    "char = generate_char_seq(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50365, 189)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = pad_sequences(words,padding='post')\n",
    "depends = pad_sequences(depends,padding='post')\n",
    "labels = pad_sequences(labels,padding='post')\n",
    "words.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "train_X, test_X, train_Y, test_Y, train_depends, test_depends, train_char, test_char = train_test_split(\n",
    "                                                                           words,\n",
    "                                                                           labels,\n",
    "                                                                           depends,\n",
    "                                                                           char,\n",
    "                                                                           test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    \n",
    "    outputs = gamma * normalized + beta\n",
    "    return outputs\n",
    "\n",
    "def multihead_attn(queries, keys, q_masks, k_masks, future_binding, num_units, num_heads):\n",
    "    \n",
    "    T_q = tf.shape(queries)[1]                                      \n",
    "    T_k = tf.shape(keys)[1]                  \n",
    "\n",
    "    Q = tf.layers.dense(queries, num_units, name='Q')                              \n",
    "    K_V = tf.layers.dense(keys, 2*num_units, name='K_V')    \n",
    "    K, V = tf.split(K_V, 2, -1)        \n",
    "\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0)                         \n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0)                    \n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0)                      \n",
    "\n",
    "    align = tf.matmul(Q_, tf.transpose(K_, [0,2,1]))                      \n",
    "    align = align / np.sqrt(K_.get_shape().as_list()[-1])                 \n",
    "\n",
    "    paddings = tf.fill(tf.shape(align), 0.0)                   \n",
    "\n",
    "    key_masks = k_masks                                                 \n",
    "    key_masks = tf.tile(key_masks, [num_heads, 1])                       \n",
    "    key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, T_q, 1])            \n",
    "    align = tf.where(tf.equal(key_masks, 0), paddings, align)       \n",
    "\n",
    "    if future_binding:\n",
    "        lower_tri = tf.ones([T_q, T_k])                                          \n",
    "        lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()  \n",
    "        masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0], 1, 1]) \n",
    "        align = tf.where(tf.equal(masks, 0), paddings, align)                      \n",
    "    \n",
    "    align = tf.nn.softmax(align)                                            \n",
    "    query_masks = tf.to_float(q_masks)                                             \n",
    "    query_masks = tf.tile(query_masks, [num_heads, 1])                             \n",
    "    query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, T_k])            \n",
    "    align *= query_masks\n",
    "    outputs = tf.matmul(align, V_)                                                 \n",
    "    outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2)             \n",
    "    outputs += queries                                                             \n",
    "    outputs = layer_norm(outputs)                                                 \n",
    "    return outputs\n",
    "\n",
    "\n",
    "def pointwise_feedforward(inputs, hidden_units, activation=None):\n",
    "    outputs = tf.layers.dense(inputs, 4*hidden_units, activation=activation)\n",
    "    outputs = tf.layers.dense(outputs, hidden_units, activation=None)\n",
    "    outputs += inputs\n",
    "    outputs = layer_norm(outputs)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def learned_position_encoding(inputs, mask, embed_dim):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    outputs = tf.range(tf.shape(inputs)[1])                # (T_q)\n",
    "    outputs = tf.expand_dims(outputs, 0)                   # (1, T_q)\n",
    "    outputs = tf.tile(outputs, [tf.shape(inputs)[0], 1])   # (N, T_q)\n",
    "    outputs = embed_seq(outputs, T, embed_dim, zero_pad=False, scale=False)\n",
    "    return tf.expand_dims(tf.to_float(mask), -1) * outputs\n",
    "\n",
    "\n",
    "def sinusoidal_position_encoding(inputs, mask, repr_dim):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1]) * tf.expand_dims(tf.to_float(mask), -1)\n",
    "\n",
    "def label_smoothing(inputs, epsilon=0.1):\n",
    "    C = inputs.get_shape().as_list()[-1]\n",
    "    return ((1 - epsilon) * inputs) + (epsilon / C)\n",
    "\n",
    "\n",
    "class CRF:\n",
    "    def __init__(self,\n",
    "                 dim_word,\n",
    "                 dim_char,\n",
    "                 dropout,\n",
    "                 learning_rate,\n",
    "                 hidden_size_char,\n",
    "                 hidden_size_word,\n",
    "                 maxlen,\n",
    "                 num_blocks = 2,\n",
    "                 num_heads = 8,\n",
    "                 min_freq = 50):\n",
    "        \n",
    "        self.word_ids = tf.placeholder(tf.int32, shape = [None, None])\n",
    "        self.char_ids = tf.placeholder(tf.int32, shape = [None, None, None])\n",
    "        self.labels = tf.placeholder(tf.int32, shape = [None, None])\n",
    "        self.depends = tf.placeholder(tf.int32, shape = [None, None])\n",
    "        self.maxlen = tf.shape(self.word_ids)[1]\n",
    "        self.lengths = tf.count_nonzero(self.word_ids, 1)\n",
    "        batch_size = tf.shape(self.word_ids)[0]\n",
    "        \n",
    "        self.word_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(word2idx), dim_word], stddev = 1.0 / np.sqrt(dim_word)\n",
    "            )\n",
    "        )\n",
    "        self.char_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(char2idx), dim_char], stddev = 1.0 / np.sqrt(dim_char)\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        word_embedded = tf.nn.embedding_lookup(\n",
    "            self.word_embeddings, self.word_ids\n",
    "        )\n",
    "        char_embedded = tf.nn.embedding_lookup(\n",
    "            self.char_embeddings, self.char_ids\n",
    "        )\n",
    "        s = tf.shape(char_embedded)\n",
    "        char_embedded = tf.reshape(\n",
    "            char_embedded, shape = [s[0] * s[1], s[-2], dim_char]\n",
    "        )\n",
    "        reshape_char = tf.reshape(self.char_ids, shape = [s[0] * s[1], s[-2]])\n",
    "        char_masked = tf.sign(reshape_char)\n",
    "        char_embedded += sinusoidal_position_encoding(reshape_char, char_masked, dim_char)\n",
    "        for i in range(num_blocks):\n",
    "            with tf.variable_scope('char_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                char_embedded = multihead_attn(queries = char_embedded,\n",
    "                                                 keys = char_embedded,\n",
    "                                                 q_masks = char_masked,\n",
    "                                                 k_masks = char_masked,\n",
    "                                                 future_binding = False,\n",
    "                                                 num_units = dim_char,\n",
    "                                                 num_heads = num_heads)\n",
    "            with tf.variable_scope('char_feedforward_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                char_embedded = pointwise_feedforward(char_embedded,\n",
    "                                                    dim_char,\n",
    "                                                    activation = tf.nn.relu)\n",
    "        output = tf.reshape(\n",
    "            char_embedded[:, -1], shape = [s[0], s[1], 2 * hidden_size_char]\n",
    "        )\n",
    "        \n",
    "        decoder_embedded = tf.concat([word_embedded, output], axis = -1)\n",
    "        decoder_embedded = tf.layers.dense(word_embedded, dim_char)\n",
    "        de_masks = tf.sign(self.word_ids)\n",
    "        \n",
    "        decoder_embedded += sinusoidal_position_encoding(self.word_ids, de_masks, dim_char)\n",
    "        \n",
    "        for i in range(num_blocks):\n",
    "            with tf.variable_scope('word_char_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                decoder_embedded = multihead_attn(queries = decoder_embedded,\n",
    "                                         keys = decoder_embedded,\n",
    "                                         q_masks = de_masks,\n",
    "                                         k_masks = de_masks,\n",
    "                                         future_binding = True,\n",
    "                                         num_units = dim_char,\n",
    "                                         num_heads = num_heads)\n",
    "                \n",
    "            with tf.variable_scope('word_char_attention_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                decoder_embedded = multihead_attn(queries = decoder_embedded,\n",
    "                                         keys = output,\n",
    "                                         q_masks = de_masks,\n",
    "                                         k_masks = de_masks,\n",
    "                                         future_binding = False,\n",
    "                                         num_units = dim_char,\n",
    "                                         num_heads = num_heads)\n",
    "            \n",
    "            with tf.variable_scope('word_feedforward_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                decoder_embedded = pointwise_feedforward(decoder_embedded,\n",
    "                                                    dim_char,\n",
    "                                            activation = tf.nn.relu)\n",
    "                \n",
    "        logits = tf.layers.dense(decoder_embedded, len(idx2tag))\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            logits, self.labels, self.lengths\n",
    "        )\n",
    "        \n",
    "        tag_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(idx2tag), dim_char], stddev = 1.0 / np.sqrt(dim_char)\n",
    "            )\n",
    "        )\n",
    "        logits_max = tf.argmax(logits,axis=2,output_type=tf.int32)\n",
    "        lookup_logits = tf.nn.embedding_lookup(\n",
    "            tag_embeddings, logits_max\n",
    "        )\n",
    "        \n",
    "        lookup_logits += sinusoidal_position_encoding(logits_max, de_masks, dim_char)\n",
    "        \n",
    "        for i in range(num_blocks):\n",
    "            with tf.variable_scope('depend_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                lookup_logits = multihead_attn(queries = lookup_logits,\n",
    "                                         keys = lookup_logits,\n",
    "                                         q_masks = de_masks,\n",
    "                                         k_masks = de_masks,\n",
    "                                         future_binding = True,\n",
    "                                         num_units = dim_char,\n",
    "                                         num_heads = num_heads)\n",
    "                \n",
    "            with tf.variable_scope('depend_attention_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                lookup_logits = multihead_attn(queries = lookup_logits,\n",
    "                                         keys = decoder_embedded,\n",
    "                                         q_masks = de_masks,\n",
    "                                         k_masks = de_masks,\n",
    "                                         future_binding = False,\n",
    "                                         num_units = dim_char,\n",
    "                                         num_heads = num_heads)\n",
    "            \n",
    "            with tf.variable_scope('depend_feedforward_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                lookup_logits = pointwise_feedforward(lookup_logits,\n",
    "                                                    dim_char,\n",
    "                                            activation = tf.nn.relu)\n",
    "        \n",
    "        cast_mask = tf.cast(tf.sequence_mask(self.lengths + 1, maxlen = maxlen), dtype = tf.float32)\n",
    "        cast_mask = tf.tile(tf.expand_dims(cast_mask,axis=1),[1,self.maxlen,1]) * 10\n",
    "        \n",
    "        logits_depends = tf.layers.dense(lookup_logits, maxlen)\n",
    "        logits_depends = tf.multiply(logits_depends, cast_mask)\n",
    "        \n",
    "        with tf.variable_scope(\"depends\"):\n",
    "            log_likelihood_depends, transition_params_depends = tf.contrib.crf.crf_log_likelihood(\n",
    "                logits_depends, self.depends, self.lengths\n",
    "            )\n",
    "            \n",
    "        self.cost = tf.reduce_mean(-log_likelihood) + tf.reduce_mean(-log_likelihood_depends)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.tags_seq, _ = tf.contrib.crf.crf_decode(\n",
    "            logits, transition_params, self.lengths\n",
    "        )\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'logits')\n",
    "        \n",
    "        self.tags_seq_depends, _ = tf.contrib.crf.crf_decode(\n",
    "            logits_depends, transition_params_depends, self.lengths\n",
    "        )\n",
    "        self.tags_seq_depends = tf.identity(self.tags_seq_depends, name = 'logits_depends')\n",
    "\n",
    "        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n",
    "        mask_label = tf.boolean_mask(self.labels, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.tags_seq_depends, mask)\n",
    "        mask_label = tf.boolean_mask(self.depends, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "dim_word = 128\n",
    "dim_char = 256\n",
    "dropout = 0.8\n",
    "learning_rate = 1e-3\n",
    "hidden_size_char = 128\n",
    "hidden_size_word = 64\n",
    "batch_size = 8\n",
    "\n",
    "model = CRF(dim_word = dim_word,\n",
    "            dim_char = dim_char,\n",
    "            dropout = dropout,\n",
    "            learning_rate = learning_rate,\n",
    "            hidden_size_char = hidden_size_char,\n",
    "            hidden_size_word = hidden_size_word,\n",
    "           maxlen = words.shape[1])\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:18<00:00,  1.51it/s, accuracy=0.756, accuracy_depends=0.524, cost=51.9]   \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.17it/s, accuracy=0.707, accuracy_depends=0.515, cost=53.7]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3971.3269250392914\n",
      "epoch: 0, training loss: 74.662333, training acc: 0.687690, training depends: 0.423968, valid loss: 53.324856, valid acc: 0.745969, valid depends: 0.509276\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:24<00:00,  1.50it/s, accuracy=0.815, accuracy_depends=0.673, cost=37.4]\n",
      "test minibatch loop: 100%|██████████| 630/630 [02:51<00:00,  4.11it/s, accuracy=0.717, accuracy_depends=0.566, cost=44.6]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3975.9911386966705\n",
      "epoch: 1, training loss: 41.632860, training acc: 0.800347, training depends: 0.614527, valid loss: 38.777110, valid acc: 0.807298, valid depends: 0.642584\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:28<00:00,  1.51it/s, accuracy=0.81, accuracy_depends=0.726, cost=32.1] \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.12it/s, accuracy=0.838, accuracy_depends=0.677, cost=34.6]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3980.6265354156494\n",
      "epoch: 2, training loss: 34.020197, training acc: 0.828245, training depends: 0.679861, valid loss: 33.156008, valid acc: 0.823669, valid depends: 0.699404\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:29<00:00,  1.50it/s, accuracy=0.839, accuracy_depends=0.75, cost=28.7] \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:51<00:00,  4.12it/s, accuracy=0.808, accuracy_depends=0.717, cost=36.1]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3980.9095969200134\n",
      "epoch: 3, training loss: 28.242658, training acc: 0.845301, training depends: 0.740665, valid loss: 28.623581, valid acc: 0.831968, valid depends: 0.757451\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:32<00:00,  1.51it/s, accuracy=0.839, accuracy_depends=0.78, cost=27]   \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.19it/s, accuracy=0.788, accuracy_depends=0.828, cost=25.5]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3985.202503681183\n",
      "epoch: 4, training loss: 23.337289, training acc: 0.859588, training depends: 0.795232, valid loss: 25.202329, valid acc: 0.840135, valid depends: 0.799228\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:33<00:00,  1.51it/s, accuracy=0.869, accuracy_depends=0.821, cost=21.9]\n",
      "test minibatch loop: 100%|██████████| 630/630 [02:51<00:00,  4.14it/s, accuracy=0.838, accuracy_depends=0.768, cost=23.9]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3985.2355260849\n",
      "epoch: 5, training loss: 18.881109, training acc: 0.873684, training depends: 0.846420, valid loss: 22.490008, valid acc: 0.849709, valid depends: 0.828853\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:32<00:00,  1.50it/s, accuracy=0.863, accuracy_depends=0.827, cost=21.8]\n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.14it/s, accuracy=0.848, accuracy_depends=0.838, cost=20.6]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3985.4282426834106\n",
      "epoch: 6, training loss: 15.691809, training acc: 0.885103, training depends: 0.882228, valid loss: 19.544368, valid acc: 0.861741, valid depends: 0.863059\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:33<00:00,  1.50it/s, accuracy=0.869, accuracy_depends=0.887, cost=17.2]\n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.10it/s, accuracy=0.778, accuracy_depends=0.879, cost=26.3]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3986.3802030086517\n",
      "epoch: 7, training loss: 13.322382, training acc: 0.895488, training depends: 0.906931, valid loss: 18.686160, valid acc: 0.859560, valid depends: 0.879505\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:32<00:00,  1.50it/s, accuracy=0.893, accuracy_depends=0.893, cost=15.4]\n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.12it/s, accuracy=0.848, accuracy_depends=0.848, cost=21.4]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3984.483060359955\n",
      "epoch: 8, training loss: 11.466844, training acc: 0.906599, training depends: 0.924221, valid loss: 16.073830, valid acc: 0.877447, valid depends: 0.899887\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:33<00:00,  1.50it/s, accuracy=0.899, accuracy_depends=0.905, cost=15.3]\n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.08it/s, accuracy=0.838, accuracy_depends=0.848, cost=20.1]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3986.1450822353363\n",
      "epoch: 9, training loss: 9.785712, training acc: 0.918415, training depends: 0.937485, valid loss: 16.119294, valid acc: 0.880889, valid depends: 0.899497\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "for e in range(10):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss, train_acc_depends, test_acc_depends = 0, 0, 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = train_X[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_char = train_char[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_y = train_Y[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_depends = train_depends[i : min(i + batch_size, train_X.shape[0])]\n",
    "        acc_depends, acc, cost, _ = sess.run(\n",
    "            [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "                model.labels: batch_y,\n",
    "                model.depends: batch_depends\n",
    "            },\n",
    "        )\n",
    "        #assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        train_acc_depends += acc_depends\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = test_X[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_char = test_char[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_depends = test_depends[i : min(i + batch_size, test_X.shape[0])]\n",
    "        acc_depends, acc, cost = sess.run(\n",
    "            [model.accuracy_depends, model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "                model.labels: batch_y,\n",
    "                model.depends: batch_depends\n",
    "            },\n",
    "        )\n",
    "        #assert not np.isnan(cost)\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        test_acc_depends += acc_depends\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    train_loss /= len(train_X) / batch_size\n",
    "    train_acc /= len(train_X) / batch_size\n",
    "    train_acc_depends /= len(train_X) / batch_size\n",
    "    test_loss /= len(test_X) / batch_size\n",
    "    test_acc /= len(test_X) / batch_size\n",
    "    test_acc_depends /= len(test_X) / batch_size\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "        % (e, train_loss, train_acc, train_acc_depends, test_loss, test_acc, test_acc_depends)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:34<00:00,  1.50it/s, accuracy=0.946, accuracy_depends=0.923, cost=9.98] \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.13it/s, accuracy=0.879, accuracy_depends=0.899, cost=16.5]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3986.8803112506866\n",
      "epoch: 0, training loss: 4.550854, training acc: 0.962310, training depends: 0.971532, valid loss: 12.594443, valid acc: 0.914797, valid depends: 0.934598\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:33<00:00,  1.51it/s, accuracy=0.94, accuracy_depends=0.946, cost=8.71]  \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.17it/s, accuracy=0.859, accuracy_depends=0.899, cost=21.7]\n",
      "train minibatch loop:   0%|          | 0/5666 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3985.9828474521637\n",
      "epoch: 1, training loss: 5.258419, training acc: 0.958980, training depends: 0.965066, valid loss: 13.166835, valid acc: 0.907223, valid depends: 0.925843\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 5666/5666 [1:03:32<00:00,  1.49it/s, accuracy=0.964, accuracy_depends=0.946, cost=6.36] \n",
      "test minibatch loop: 100%|██████████| 630/630 [02:52<00:00,  4.13it/s, accuracy=0.899, accuracy_depends=0.909, cost=19.7]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3985.3402976989746\n",
      "epoch: 2, training loss: 3.728318, training acc: 0.968881, training depends: 0.976212, valid loss: 11.758984, valid acc: 0.920203, valid depends: 0.942277\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "for e in range(3):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss, train_acc_depends, test_acc_depends = 0, 0, 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = train_X[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_char = train_char[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_y = train_Y[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_depends = train_depends[i : min(i + batch_size, train_X.shape[0])]\n",
    "        acc_depends, acc, cost, _ = sess.run(\n",
    "            [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "                model.labels: batch_y,\n",
    "                model.depends: batch_depends\n",
    "            },\n",
    "        )\n",
    "        #assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        train_acc_depends += acc_depends\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = test_X[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_char = test_char[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_depends = test_depends[i : min(i + batch_size, test_X.shape[0])]\n",
    "        acc_depends, acc, cost = sess.run(\n",
    "            [model.accuracy_depends, model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "                model.labels: batch_y,\n",
    "                model.depends: batch_depends\n",
    "            },\n",
    "        )\n",
    "        #assert not np.isnan(cost)\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        test_acc_depends += acc_depends\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    train_loss /= len(train_X) / batch_size\n",
    "    train_acc /= len(train_X) / batch_size\n",
    "    train_acc_depends /= len(train_X) / batch_size\n",
    "    test_loss /= len(test_X) / batch_size\n",
    "    test_acc /= len(test_X) / batch_size\n",
    "    test_acc_depends /= len(test_X) / batch_size\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "        % (e, train_loss, train_acc, train_acc_depends, test_loss, test_acc, test_acc_depends)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'Placeholder_3',\n",
       " 'Variable',\n",
       " 'Variable_1',\n",
       " 'char_0/Q/kernel',\n",
       " 'char_0/Q/bias',\n",
       " 'char_0/K_V/kernel',\n",
       " 'char_0/K_V/bias',\n",
       " 'char_0/gamma',\n",
       " 'char_feedforward_0/dense/kernel',\n",
       " 'char_feedforward_0/dense/bias',\n",
       " 'char_feedforward_0/dense_1/kernel',\n",
       " 'char_feedforward_0/dense_1/bias',\n",
       " 'char_feedforward_0/gamma',\n",
       " 'char_1/Q/kernel',\n",
       " 'char_1/Q/bias',\n",
       " 'char_1/K_V/kernel',\n",
       " 'char_1/K_V/bias',\n",
       " 'char_1/gamma',\n",
       " 'char_feedforward_1/dense/kernel',\n",
       " 'char_feedforward_1/dense/bias',\n",
       " 'char_feedforward_1/dense_1/kernel',\n",
       " 'char_feedforward_1/dense_1/bias',\n",
       " 'char_feedforward_1/gamma',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'word_char_0/Q/kernel',\n",
       " 'word_char_0/Q/bias',\n",
       " 'word_char_0/K_V/kernel',\n",
       " 'word_char_0/K_V/bias',\n",
       " 'word_char_0/gamma',\n",
       " 'word_char_attention_0/Q/kernel',\n",
       " 'word_char_attention_0/Q/bias',\n",
       " 'word_char_attention_0/K_V/kernel',\n",
       " 'word_char_attention_0/K_V/bias',\n",
       " 'word_char_attention_0/gamma',\n",
       " 'word_feedforward_0/dense/kernel',\n",
       " 'word_feedforward_0/dense/bias',\n",
       " 'word_feedforward_0/dense_1/kernel',\n",
       " 'word_feedforward_0/dense_1/bias',\n",
       " 'word_feedforward_0/gamma',\n",
       " 'word_char_1/Q/kernel',\n",
       " 'word_char_1/Q/bias',\n",
       " 'word_char_1/K_V/kernel',\n",
       " 'word_char_1/K_V/bias',\n",
       " 'word_char_1/gamma',\n",
       " 'word_char_attention_1/Q/kernel',\n",
       " 'word_char_attention_1/Q/bias',\n",
       " 'word_char_attention_1/K_V/kernel',\n",
       " 'word_char_attention_1/K_V/bias',\n",
       " 'word_char_attention_1/gamma',\n",
       " 'word_feedforward_1/dense/kernel',\n",
       " 'word_feedforward_1/dense/bias',\n",
       " 'word_feedforward_1/dense_1/kernel',\n",
       " 'word_feedforward_1/dense_1/bias',\n",
       " 'word_feedforward_1/gamma',\n",
       " 'dense_1/kernel',\n",
       " 'dense_1/bias',\n",
       " 'transitions',\n",
       " 'Variable_2',\n",
       " 'depend_0/Q/kernel',\n",
       " 'depend_0/Q/bias',\n",
       " 'depend_0/K_V/kernel',\n",
       " 'depend_0/K_V/bias',\n",
       " 'depend_0/gamma',\n",
       " 'depend_attention_0/Q/kernel',\n",
       " 'depend_attention_0/Q/bias',\n",
       " 'depend_attention_0/K_V/kernel',\n",
       " 'depend_attention_0/K_V/bias',\n",
       " 'depend_attention_0/gamma',\n",
       " 'depend_feedforward_0/dense/kernel',\n",
       " 'depend_feedforward_0/dense/bias',\n",
       " 'depend_feedforward_0/dense_1/kernel',\n",
       " 'depend_feedforward_0/dense_1/bias',\n",
       " 'depend_feedforward_0/gamma',\n",
       " 'depend_1/Q/kernel',\n",
       " 'depend_1/Q/bias',\n",
       " 'depend_1/K_V/kernel',\n",
       " 'depend_1/K_V/bias',\n",
       " 'depend_1/gamma',\n",
       " 'depend_attention_1/Q/kernel',\n",
       " 'depend_attention_1/Q/bias',\n",
       " 'depend_attention_1/K_V/kernel',\n",
       " 'depend_attention_1/K_V/bias',\n",
       " 'depend_attention_1/gamma',\n",
       " 'depend_feedforward_1/dense/kernel',\n",
       " 'depend_feedforward_1/dense/bias',\n",
       " 'depend_feedforward_1/dense_1/kernel',\n",
       " 'depend_feedforward_1/dense_1/bias',\n",
       " 'depend_feedforward_1/gamma',\n",
       " 'dense_2/kernel',\n",
       " 'dense_2/bias',\n",
       " 'depends/transitions',\n",
       " 'logits',\n",
       " 'logits_depends']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'attention/model.ckpt')\n",
    "\n",
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'logits_depends' in n.name\n",
    "        or 'alphas' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'OptimizeLoss' not in n.name\n",
    "        and 'Global_Step' not in n.name\n",
    "        and 'Epoch_Step' not in n.name\n",
    "        and 'learning_rate' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq, deps = sess.run([model.tags_seq, model.tags_seq_depends],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "            },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 630/630 [02:47<00:00,  4.18it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y, real_depends, predict_depends = [], [], [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_X), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    batch_x = test_X[i : min(i + batch_size, test_X.shape[0])]\n",
    "    batch_char = test_char[i : min(i + batch_size, test_X.shape[0])]\n",
    "    batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "    batch_depends = test_depends[i : min(i + batch_size, test_X.shape[0])]\n",
    "    seq, deps = sess.run([model.tags_seq, model.tags_seq_depends],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "            },\n",
    "    )\n",
    "    predicted = pred2label(seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)\n",
    "    \n",
    "    real_depends.extend(batch_depends.tolist())\n",
    "    predict_depends.extend(deps.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD     1.0000    1.0000    1.0000    841796\n",
      "          acl     0.8768    0.8849    0.8809      3016\n",
      "        advcl     0.8290    0.7943    0.8113      1196\n",
      "       advmod     0.9043    0.9163    0.9102      4754\n",
      "         amod     0.9121    0.8773    0.8943      4149\n",
      "        appos     0.8934    0.8983    0.8958      2547\n",
      "          aux     1.0000    1.0000    1.0000         6\n",
      "         case     0.9593    0.9670    0.9631     10888\n",
      "           cc     0.9523    0.9606    0.9564      3198\n",
      "        ccomp     0.7984    0.7385    0.7673       413\n",
      "     compound     0.8677    0.8956    0.8815      6679\n",
      "compound:plur     0.9073    0.9255    0.9163       550\n",
      "         conj     0.8625    0.9330    0.8964      4162\n",
      "          cop     0.9296    0.9679    0.9484       996\n",
      "        csubj     0.9000    0.4091    0.5625        22\n",
      "   csubj:pass     0.8462    0.8462    0.8462        13\n",
      "          dep     0.8274    0.7377    0.7800       507\n",
      "          det     0.8897    0.9196    0.9044      4094\n",
      "        fixed     0.8851    0.7966    0.8385       580\n",
      "         flat     0.9468    0.9198    0.9331     10333\n",
      "         iobj     1.0000    0.6000    0.7500        20\n",
      "         mark     0.8535    0.8447    0.8491      1359\n",
      "         nmod     0.8749    0.8907    0.8827      4107\n",
      "        nsubj     0.8746    0.8881    0.8813      6471\n",
      "   nsubj:pass     0.8478    0.7116    0.7738      1949\n",
      "       nummod     0.9568    0.9524    0.9546      3884\n",
      "          obj     0.9082    0.8946    0.9013      5274\n",
      "          obl     0.9203    0.8854    0.9025      5740\n",
      "    parataxis     0.7980    0.7980    0.7980       391\n",
      "        punct     0.9933    0.9957    0.9945     16561\n",
      "         root     0.8974    0.9200    0.9085      5037\n",
      "        xcomp     0.8580    0.8593    0.8587      1301\n",
      "\n",
      "  avg / total     0.9906    0.9906    0.9906    951993\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(np.array(real_Y).ravel(), np.array(predict_Y).ravel(), digits = 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0     1.0000    1.0000    1.0000    841796\n",
      "          1     0.9486    0.9277    0.9381      5037\n",
      "          2     0.9157    0.9547    0.9348      4325\n",
      "          3     0.9505    0.9137    0.9318      4856\n",
      "          4     0.9439    0.9311    0.9374      6309\n",
      "          5     0.9422    0.9396    0.9409      6540\n",
      "          6     0.9314    0.9516    0.9414      5697\n",
      "          7     0.9468    0.9461    0.9464      5414\n",
      "          8     0.9524    0.9394    0.9458      5559\n",
      "          9     0.9432    0.9421    0.9427      5028\n",
      "         10     0.9308    0.9544    0.9425      4300\n",
      "         11     0.9623    0.9323    0.9471      4358\n",
      "         12     0.9449    0.9493    0.9471      3903\n",
      "         13     0.9338    0.9442    0.9390      3497\n",
      "         14     0.9444    0.9475    0.9459      3445\n",
      "         15     0.9445    0.9487    0.9466      3177\n",
      "         16     0.9411    0.9589    0.9500      3068\n",
      "         17     0.9350    0.9589    0.9468      2774\n",
      "         18     0.9527    0.9352    0.9439      2499\n",
      "         19     0.9767    0.9207    0.9478      2319\n",
      "         20     0.9445    0.9558    0.9501      2013\n",
      "         21     0.9321    0.9374    0.9347      2124\n",
      "         22     0.9337    0.9423    0.9380      1749\n",
      "         23     0.9508    0.9175    0.9339      1685\n",
      "         24     0.9608    0.9240    0.9421      1540\n",
      "         25     0.8654    0.9661    0.9130      1358\n",
      "         26     0.9511    0.9245    0.9376      1179\n",
      "         27     0.9416    0.9367    0.9392      1154\n",
      "         28     0.8961    0.9549    0.9245       975\n",
      "         29     0.9260    0.9383    0.9321      1054\n",
      "         30     0.9342    0.9551    0.9445      1025\n",
      "         31     0.9482    0.9146    0.9311       761\n",
      "         32     0.9549    0.9126    0.9333       835\n",
      "         33     0.9235    0.9506    0.9368       749\n",
      "         34     0.9492    0.9465    0.9478       710\n",
      "         35     0.9323    0.9649    0.9483       599\n",
      "         36     0.9750    0.9458    0.9602       535\n",
      "         37     0.9363    0.9620    0.9490       474\n",
      "         38     0.9099    0.9815    0.9443       432\n",
      "         39     0.9462    0.9342    0.9401       395\n",
      "         40     0.9170    0.9535    0.9349       452\n",
      "         41     0.9446    0.9214    0.9328       407\n",
      "         42     0.9452    0.9452    0.9452       292\n",
      "         43     0.9731    0.9031    0.9368       320\n",
      "         44     0.9030    0.9767    0.9384       343\n",
      "         45     0.9343    0.9812    0.9572       319\n",
      "         46     0.9943    0.7955    0.8838       220\n",
      "         47     0.9420    0.9684    0.9550       285\n",
      "         48     0.9160    0.9745    0.9443       235\n",
      "         49     0.9113    0.9893    0.9487       187\n",
      "         50     0.9568    0.8636    0.9078       154\n",
      "         51     0.9706    0.9538    0.9621       173\n",
      "         52     0.9554    0.9934    0.9740       151\n",
      "         53     0.9116    0.9515    0.9311       206\n",
      "         54     0.9008    0.9833    0.9402       120\n",
      "         55     0.9371    0.9371    0.9371       159\n",
      "         56     0.9179    0.9535    0.9354       129\n",
      "         57     0.9091    0.8824    0.8955       102\n",
      "         58     0.9350    0.9127    0.9237       126\n",
      "         59     0.9725    0.7910    0.8724       134\n",
      "         60     0.9576    0.9826    0.9700       115\n",
      "         61     0.9200    0.9485    0.9340        97\n",
      "         62     0.9200    0.9079    0.9139        76\n",
      "         63     0.9551    0.9770    0.9659        87\n",
      "         64     0.9878    0.9310    0.9586        87\n",
      "         65     0.9103    0.9861    0.9467        72\n",
      "         66     0.9474    0.9863    0.9664        73\n",
      "         67     1.0000    0.9667    0.9831        60\n",
      "         68     0.9855    0.8831    0.9315        77\n",
      "         69     0.8889    0.9231    0.9057        52\n",
      "         70     0.9524    1.0000    0.9756        80\n",
      "         71     0.9241    0.9605    0.9419        76\n",
      "         72     0.9870    0.9870    0.9870        77\n",
      "         73     0.9531    1.0000    0.9760        61\n",
      "         74     1.0000    0.9667    0.9831        30\n",
      "         75     0.9412    1.0000    0.9697        64\n",
      "         76     1.0000    0.8571    0.9231        28\n",
      "         77     0.9487    1.0000    0.9737        37\n",
      "         78     0.9677    0.9677    0.9677        31\n",
      "         79     1.0000    1.0000    1.0000        25\n",
      "         80     1.0000    0.9348    0.9663        46\n",
      "         81     1.0000    0.9756    0.9877        41\n",
      "         82     1.0000    0.9302    0.9639        43\n",
      "         83     0.9474    1.0000    0.9730        18\n",
      "         84     0.8846    1.0000    0.9388        23\n",
      "         85     0.9583    1.0000    0.9787        23\n",
      "         86     1.0000    0.8636    0.9268        44\n",
      "         87     1.0000    1.0000    1.0000        10\n",
      "         88     0.9412    0.9412    0.9412        17\n",
      "         89     1.0000    0.8750    0.9333         8\n",
      "         90     0.9167    0.9565    0.9362        23\n",
      "         91     1.0000    1.0000    1.0000        15\n",
      "         92     1.0000    1.0000    1.0000        34\n",
      "         93     0.8571    1.0000    0.9231         6\n",
      "         94     0.9231    1.0000    0.9600        12\n",
      "         95     1.0000    1.0000    1.0000         9\n",
      "         96     1.0000    0.9333    0.9655        15\n",
      "         97     1.0000    1.0000    1.0000        30\n",
      "         98     1.0000    1.0000    1.0000         8\n",
      "         99     1.0000    0.9200    0.9583        25\n",
      "        100     0.8571    1.0000    0.9231         6\n",
      "        101     1.0000    0.9744    0.9870        39\n",
      "        102     1.0000    1.0000    1.0000         7\n",
      "        103     0.8889    1.0000    0.9412        16\n",
      "        104     1.0000    0.9500    0.9744        20\n",
      "        105     1.0000    0.9000    0.9474        10\n",
      "        106     0.9500    1.0000    0.9744        19\n",
      "        107     0.7500    1.0000    0.8571        27\n",
      "        108     1.0000    1.0000    1.0000        15\n",
      "        109     1.0000    1.0000    1.0000         3\n",
      "        110     1.0000    1.0000    1.0000        14\n",
      "        111     1.0000    1.0000    1.0000         9\n",
      "        112     0.9474    1.0000    0.9730        18\n",
      "        113     0.8571    1.0000    0.9231         6\n",
      "        114     1.0000    1.0000    1.0000        10\n",
      "        115     1.0000    1.0000    1.0000         7\n",
      "        116     1.0000    0.9375    0.9677        16\n",
      "        117     1.0000    0.5000    0.6667         2\n",
      "        118     1.0000    1.0000    1.0000        12\n",
      "        119     1.0000    1.0000    1.0000         4\n",
      "        120     1.0000    0.9231    0.9600        13\n",
      "        121     1.0000    1.0000    1.0000         6\n",
      "        122     1.0000    1.0000    1.0000         3\n",
      "        123     1.0000    0.8333    0.9091         6\n",
      "        124     1.0000    1.0000    1.0000         2\n",
      "        125     1.0000    1.0000    1.0000         2\n",
      "        126     0.8846    1.0000    0.9388        23\n",
      "        127     1.0000    1.0000    1.0000         6\n",
      "        128     1.0000    1.0000    1.0000         5\n",
      "        129     1.0000    0.8333    0.9091         6\n",
      "        130     1.0000    1.0000    1.0000        12\n",
      "        131     1.0000    0.7143    0.8333         7\n",
      "        132     1.0000    1.0000    1.0000         2\n",
      "        133     1.0000    1.0000    1.0000         4\n",
      "        134     0.9000    0.9000    0.9000        10\n",
      "        135     0.8571    1.0000    0.9231         6\n",
      "        136     1.0000    1.0000    1.0000         7\n",
      "        137     1.0000    1.0000    1.0000         8\n",
      "        138     1.0000    1.0000    1.0000        12\n",
      "        139     1.0000    1.0000    1.0000         1\n",
      "        140     1.0000    1.0000    1.0000         2\n",
      "        141     1.0000    1.0000    1.0000         2\n",
      "        142     1.0000    1.0000    1.0000         4\n",
      "        144     1.0000    1.0000    1.0000         4\n",
      "        146     1.0000    1.0000    1.0000         3\n",
      "        147     1.0000    1.0000    1.0000         7\n",
      "        149     1.0000    1.0000    1.0000         2\n",
      "        150     1.0000    1.0000    1.0000         2\n",
      "        151     1.0000    1.0000    1.0000         2\n",
      "        152     1.0000    1.0000    1.0000         1\n",
      "        153     1.0000    1.0000    1.0000         1\n",
      "        154     1.0000    1.0000    1.0000         2\n",
      "        156     1.0000    1.0000    1.0000         6\n",
      "        157     1.0000    1.0000    1.0000         1\n",
      "        158     1.0000    1.0000    1.0000         5\n",
      "        159     1.0000    1.0000    1.0000         1\n",
      "        160     1.0000    1.0000    1.0000         2\n",
      "        162     0.6667    0.6667    0.6667         3\n",
      "        163     0.6667    1.0000    0.8000         2\n",
      "        164     1.0000    1.0000    1.0000         2\n",
      "        167     1.0000    0.7500    0.8571         4\n",
      "        174     1.0000    1.0000    1.0000         2\n",
      "        176     1.0000    1.0000    1.0000         4\n",
      "        177     1.0000    1.0000    1.0000         2\n",
      "        178     1.0000    1.0000    1.0000         1\n",
      "        179     1.0000    1.0000    1.0000         1\n",
      "        182     1.0000    1.0000    1.0000         4\n",
      "        183     1.0000    1.0000    1.0000         4\n",
      "\n",
      "avg / total     0.9933    0.9932    0.9932    951993\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(np.array(real_depends).ravel(), \n",
    "                            np.array(predict_depends).ravel(), digits = 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tolong', 'tangkap', 'gambar', 'kami']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'tolong tangkap gambar kami'\n",
    "\n",
    "def char_str_idx(corpus, dic, UNK = 0):\n",
    "    maxlen = max([len(i) for i in corpus])\n",
    "    X = np.zeros((len(corpus), maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i][:maxlen]):\n",
    "            val = dic[k] if k in dic else UNK\n",
    "            X[i, no] = val\n",
    "    return X\n",
    "\n",
    "def generate_char_seq(batch, UNK = 2):\n",
    "    maxlen_c = max([len(k) for k in batch])\n",
    "    x = [[len(i) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((len(batch),maxlen_c,maxlen),dtype=np.int32)\n",
    "    for i in range(len(batch)):\n",
    "        for k in range(len(batch[i])):\n",
    "            for no, c in enumerate(batch[i][k][::-1]):\n",
    "                temp[i,k,-1-no] = char2idx.get(c, UNK)\n",
    "    return temp\n",
    "\n",
    "sequence = process_string(string)\n",
    "sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_seq = char_str_idx([sequence], word2idx, 2)\n",
    "X_char_seq = generate_char_seq([sequence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4, 7)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_char_seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq, deps = sess.run([model.tags_seq, model.tags_seq_depends],\n",
    "        feed_dict={model.word_ids:X_seq,\n",
    "                  model.char_ids:X_char_seq})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 3, 3], dtype=int32)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deps[0] - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['advmod', 'root', 'case', 'nmod']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[idx2tag[i] for i in seq[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('attention-is-all-you-need-dependency.json','w') as fopen:\n",
    "    fopen.write(json.dumps({'idx2tag':idx2tag,'idx2word':idx2word,\n",
    "           'word2idx':word2idx,'tag2idx':tag2idx,'char2idx':char2idx}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))\n",
    "        \n",
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from attention/model.ckpt\n",
      "INFO:tensorflow:Froze 107 variables.\n",
      "INFO:tensorflow:Converted 107 variables to const ops.\n",
      "2815 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('attention', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('attention/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1702: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[14  4  7 20]] [[3 1 4 4]]\n"
     ]
    }
   ],
   "source": [
    "word_ids = g.get_tensor_by_name('import/Placeholder:0')\n",
    "char_ids = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "tags_seq = g.get_tensor_by_name('import/logits:0')\n",
    "depends_seq = g.get_tensor_by_name('import/logits_depends:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)\n",
    "seq, deps = test_sess.run([tags_seq, depends_seq],\n",
    "            feed_dict = {\n",
    "                word_ids: X_seq,\n",
    "                char_ids: X_char_seq,\n",
    "            })\n",
    "\n",
    "print(seq,deps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'PAD',\n",
       " 1: 'nsubj',\n",
       " 2: 'cop',\n",
       " 3: 'det',\n",
       " 4: 'root',\n",
       " 5: 'nsubj:pass',\n",
       " 6: 'acl',\n",
       " 7: 'case',\n",
       " 8: 'obl',\n",
       " 9: 'flat',\n",
       " 10: 'punct',\n",
       " 11: 'appos',\n",
       " 12: 'amod',\n",
       " 13: 'compound',\n",
       " 14: 'advmod',\n",
       " 15: 'cc',\n",
       " 16: 'obj',\n",
       " 17: 'conj',\n",
       " 18: 'mark',\n",
       " 19: 'advcl',\n",
       " 20: 'nmod',\n",
       " 21: 'nummod',\n",
       " 22: 'dep',\n",
       " 23: 'xcomp',\n",
       " 24: 'ccomp',\n",
       " 25: 'parataxis',\n",
       " 26: 'compound:plur',\n",
       " 27: 'fixed',\n",
       " 28: 'aux',\n",
       " 29: 'csubj',\n",
       " 30: 'iobj',\n",
       " 31: 'csubj:pass'}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx2tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
